{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a3476d4e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import re \n",
    "import pickle \n",
    "import time \n",
    "import random \n",
    "import threading \n",
    "import pandas as pd \n",
    "import requests \n",
    "from concurrent.futures  import ThreadPoolExecutor, as_completed \n",
    "from selenium import webdriver \n",
    "from selenium.webdriver.edge.service  import Service \n",
    "from selenium.webdriver.edge.options  import Options \n",
    "from selenium.webdriver.common.by  import By \n",
    "from webdriver_manager.microsoft  import EdgeChromiumDriverManager \n",
    "from bs4 import BeautifulSoup \n",
    " \n",
    "# 全局配置 \n",
    "COOKIES_FILE = \"weibo_cookies.pkl\" \n",
    "MIN_COMMENTS = 2000 \n",
    "THREAD_WORKERS = 8 \n",
    "REQUEST_DELAY = (0.3, 1.2)  # 随机延迟区间（秒）\n",
    "HEADERS = {\n",
    "    \"User-Agent\": \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36 Edg/125.0.0.0\",\n",
    "    \"Referer\": \"https://weibo.com/\", \n",
    "    \"X-Requested-With\": \"XMLHttpRequest\"\n",
    "}\n",
    " \n",
    "# 全局状态控制 \n",
    "global_lock = threading.Lock()\n",
    "total_comments = 0 \n",
    "stop_flag = False \n",
    " \n",
    "def init_driver():\n",
    "    \"\"\"初始化无头浏览器\"\"\"\n",
    "    options = Options()\n",
    "    options.binary_location  = \"C:\\\\Program Files (x86)\\\\Microsoft\\\\Edge\\\\Application\\\\msedge.exe\" \n",
    "    options.add_argument(\"--headless=new\") \n",
    "    options.add_argument(\"--disable-gpu\") \n",
    "    options.add_argument(\"--blink-settings=imagesEnabled=false\") \n",
    "    options.add_argument(\"--log-level=3\") \n",
    "    service = Service(EdgeChromiumDriverManager().install())\n",
    "    return webdriver.Edge(service=service, options=options)\n",
    " \n",
    "def load_cookies(driver):\n",
    "    \"\"\"加载持久化Cookies\"\"\"\n",
    "    driver.get(\"https://weibo.com\") \n",
    "    time.sleep(2) \n",
    "    try:\n",
    "        with open(COOKIES_FILE, \"rb\") as f:\n",
    "            cookies = pickle.load(f) \n",
    "            for cookie in cookies:\n",
    "                if 'expiry' in cookie: del cookie['expiry']\n",
    "                driver.add_cookie(cookie) \n",
    "        print(\"Cookies加载成功\")\n",
    "    except Exception as e:\n",
    "        print(\"Cookies加载失败:\", str(e))\n",
    "        manual_login(driver)\n",
    " \n",
    "def manual_login(driver):\n",
    "    \"\"\"手动登录处理\"\"\"\n",
    "    driver.get(\"https://weibo.com/login.php\") \n",
    "    print(\"请于120秒内完成登录...\")\n",
    "    time.sleep(120) \n",
    "    with open(COOKIES_FILE, \"wb\") as f:\n",
    "        pickle.dump(driver.get_cookies(),  f)\n",
    "    print(\"Cookies已更新\")\n",
    " \n",
    "def extract_post_ids(driver, topic_url):\n",
    "    \"\"\"智能获取博文ID\"\"\"\n",
    "    driver.get(topic_url) \n",
    "    time.sleep(random.uniform(*REQUEST_DELAY)) \n",
    "    soup = BeautifulSoup(driver.page_source,  'html.parser') \n",
    "    return list(set(re.findall(r'weibo\\.com/\\d+/(\\w+)',  str(soup))))\n",
    " \n",
    "def fetch_comments(mid, uid, cookies):\n",
    "    \"\"\"高速获取评论核心函数\"\"\"\n",
    "    session = requests.Session()\n",
    "    session.headers.update(HEADERS) \n",
    "    session.cookies.update(cookies) \n",
    "    comments = []\n",
    "    max_id = ''\n",
    "    \n",
    "    while not stop_flag:\n",
    "        params = {\n",
    "            \"is_reload\": 1,\n",
    "            \"id\": mid,\n",
    "            \"is_show_bulletin\": 3,\n",
    "            \"count\": 100,  # 单次最大请求量 \n",
    "            \"uid\": uid,\n",
    "            \"max_id\": max_id,\n",
    "            \"locale\": \"zh-CN\"\n",
    "        }\n",
    "        try:\n",
    "            resp = session.get(\"https://weibo.com/ajax/statuses/buildComments\",  \n",
    "                             params=params, timeout=10)\n",
    "            if resp.status_code  != 200:\n",
    "                print(f\"请求异常: {resp.status_code}\") \n",
    "                break \n",
    "                \n",
    "            data = resp.json() \n",
    "            comments.extend([{ \n",
    "                \"user\": c.get('user',  {}).get('screen_name', ''),\n",
    "                \"text\": c.get('text_raw',  '').strip(),\n",
    "                \"time\": c.get('created_at',  ''),\n",
    "                \"mid\": mid \n",
    "            } for c in data.get('data',  [])])\n",
    "            \n",
    "            if not (max_id := data.get('max_id')): \n",
    "                break \n",
    "                \n",
    "            time.sleep(random.uniform(*REQUEST_DELAY)) \n",
    "        except Exception as e:\n",
    "            print(f\"请求失败: {str(e)}\")\n",
    "            break \n",
    "            \n",
    "    return comments \n",
    " \n",
    "def comment_processor(mid, uid, cookies):\n",
    "    \"\"\"评论处理线程\"\"\"\n",
    "    global total_comments, stop_flag \n",
    "    \n",
    "    if stop_flag:\n",
    "        return []\n",
    "        \n",
    "    comments = fetch_comments(mid, uid, cookies)\n",
    "    valid_comments = [c for c in comments if len(c['text']) > 5]  # 过滤无效评论 \n",
    "    \n",
    "    with global_lock:\n",
    "        if total_comments + len(valid_comments) >= MIN_COMMENTS:\n",
    "            valid_comments = valid_comments[:MIN_COMMENTS - total_comments]\n",
    "            stop_flag = True \n",
    "        total_comments += len(valid_comments)\n",
    "        \n",
    "    print(f\"MID:{mid} 获取{len(valid_comments)}条 (总计:{total_comments})\")\n",
    "    return valid_comments \n",
    " \n",
    "def main():\n",
    "    driver = init_driver()\n",
    "    try:\n",
    "        load_cookies(driver)\n",
    "        topic_url = input(\"请输入微博话题URL: \")\n",
    "        cookies = {c['name']:c['value'] for c in driver.get_cookies()} \n",
    "        \n",
    "        print(\"智能解析话题页...\")\n",
    "        post_ids = extract_post_ids(driver, topic_url)[:125]  # 预取125个博文ID \n",
    "        \n",
    "        with ThreadPoolExecutor(max_workers=THREAD_WORKERS) as executor:\n",
    "            futures = []\n",
    "            for mid in post_ids:\n",
    "                if stop_flag: break \n",
    "                futures.append(executor.submit( \n",
    "                    comment_processor, mid, '', cookies  # 实际场景需解析uid \n",
    "                ))\n",
    "                \n",
    "            results = []\n",
    "            for future in as_completed(futures):\n",
    "                if res := future.result(): \n",
    "                    results.extend(res) \n",
    "                if stop_flag:\n",
    "                    executor.shutdown(wait=False,  cancel_futures=True)\n",
    "                    break \n",
    "                    \n",
    "        pd.DataFrame(results).to_csv(\n",
    "            f\"微博评论_{time.strftime('%Y%m%d%H%M')}.csv\",  \n",
    "            index=False, \n",
    "            encoding='utf_8_sig'\n",
    "        )\n",
    "        print(f\"任务完成，共获取{len(results)}条有效评论\")\n",
    "        \n",
    "    finally:\n",
    "        driver.quit() \n",
    " \n",
    "if __name__ == \"__main__\":\n",
    "main()\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pytorch",
   "language": "python",
   "name": "env_name"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.21"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
